// TODO:
// Make title location lookup more efficient with hash table
// Write something to combine results. (read in line, split on space, Float.parseFloat(), sum, if sum > 0.002, overwrite line in String[] linesOut

package u1;

import static java.lang.System.*;

import java.io.IOException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.BufferedReader;
import java.io.BufferedWriter;

import java.util.Vector;

import gnu.trove.THashMap;

import javax.xml.stream.FactoryConfigurationError;
import javax.xml.stream.XMLStreamException;

import org.xml.sax.SAXException;

public class Similarity {
    public static String tfidfFile; // Defines the file that contains the output from TFIDF.tcl

    //public static float[][] similarities = new float[17770][17770]; // holds output similarities for all movies.
    public static String[] titleArray = new String[17770]; // Contains the article title corresponding to every netflix title, in lowercase

    public static Vector<Float> vectorLengths = new Vector<Float>(); // Contains the 'length' of each article as a Float. Keyed by their order in the TFIDF file. This should have around 13k entries.
    public static Vector<String> movieTitles = new Vector<String>(); // Contains the lowercased movieTitles as they are read in as a String. Keyed by their order in the TFIDF file.
    public static THashMap<String,Float> tfidfMap = new THashMap<String,Float>(); // holds TFIDF for all movies with string: title@term as the key
    //public static THashMap movieTitleMap = new THashMap(); // Indexed by lowercase movie title, key is a vector of Integers corresponding to title IDs
	
    public static void main(final String[] argv) throws SAXException, IOException, XMLStreamException, FactoryConfigurationError {
	if (argv.length != 3) {
	    err.println("USAGE: <TFIDF.txt> <titles.done> <pid [0-2]>");
	    exit(-1);
	}

	// Parse args.
	tfidfFile = argv[0]; // Needs to be static for use by the Calculators
	String titlesFile = argv[1];
	int pid = Integer.parseInt(argv[2]);

	// init similarities, titleArray
	/*	for (int i = 0; i < 17770; i++) {
	    for (int j = 0; j < 17770; j++) {
		similarities[i][j] = 0.0f;
	    }
	    }*/

	// READ IN MOVIE TITLES into titleArray
	int lineNo = 0;
	try {
	    FileReader input = new FileReader(titlesFile);
	    BufferedReader bufRead = new BufferedReader(input);
	    String line;

	    // Fill movieTitleMap
	    line = (bufRead.readLine()).toLowerCase();
	    while (line != null){
		/*	     	if (!movieTitleMap.containsKey(line)) {
		    Vector temp = new Vector();
		    temp.add(new Integer(lineNo++));
		    movieTitleMap.put(line, temp);
		} else {
		    Vector temp = (Vector)(movieTitleMap.get(line));
		    temp.add(new Integer(lineNo++));
		    movieTitleMap.put(line, temp);
		    }*/
		titleArray[lineNo++] = line.toLowerCase();
		line = bufRead.readLine();
	    }

	    bufRead.close();
	} catch (IOException e){
	    System.out.println("Error reading in titles.done file"); e.printStackTrace();
	    }
	    
	// READ IN TFIDF into tfidfMap, calculate vector lengths
	try {
	    BufferedReader bufRead = new BufferedReader(new FileReader(tfidfFile));
	    String line;
		    
	    line = bufRead.readLine();
	    while (line != null){
		String title = line.substring(0, line.indexOf("@"));
		line = line.substring(line.indexOf("@")+1, line.length());

		// each element in splitline represents a pair of term,TFIDF
		String[] splitline = line.split(":");
		double lengthAccum = 0;
		for (int i = 0; i < splitline.length; i++) {
		    String[] terms = splitline[i].split("@");
		    String index = title+"@"+terms[0];

		    float val = Float.parseFloat(terms[1]);
		    tfidfMap.put(index, new Float(val));
		    lengthAccum += val * val;
		}
		movieTitles.add(title);
		vectorLengths.add(new Float(Math.sqrt(lengthAccum)));

		line = bufRead.readLine();
	    }

	    bufRead.close();
	} catch (IOException e){
	    System.out.println("Error reading in TFIDF files");
	    e.printStackTrace();
	}
	
	// tfidfMap, movieTitleMap, vectorLengths, movieTitles all filled here.
	
	// Create 4 threads.
	Thread t[] = new Thread[4];
	for (int i = 0; i < 4; i++) {
	    // pid is 0, 1, 2. (12 threads total running across 3 processes)
	    int startRange = pid * (movieTitles.size() / 3);
	    int endRange = (pid + 1) * (movieTitles.size() / 3);
	    
	    t[i] = new Thread(new Calculator(startRange + (i*(endRange - startRange)/4), startRange + (i+1)*(endRange - startRange)/4));
	    t[i].start();
	}

	try {
	    for (int i = 0; i < 4; i++) {
		t[i].join();
	    }
	} catch (Exception e) {
	    System.out.println("Exception in the join"); e.printStackTrace();
	}

	// Print output
	/*	for (int i = 0; i < 17770; i++) {
	    String lineOut = "";
	    for (int j = 0; j < 17770; j++) {
		lineOut += similarities[i][j]+" ";
	    }
	    System.out.println(lineOut);
	    }	*/    
	
    }
    
    private static class Calculator implements Runnable {
	private int startRange = 0, endRange = 0;
	private float[]  tempSimilarity = new float[17770];
	private int completed = 0;
	BufferedWriter out;

	public void run() {
	    try {
		BufferedReader bufRead = new BufferedReader(new FileReader(tfidfFile));
		String line;    // String that holds current file line
		int lineNo = 0; // lineNo corresponds to the index in the movieTitles/vectorLengths array

		// Read in the TFIDF file line by line. If the line number is within the given range, this thread should handle it.
		// To handle it, it computes the similarity between the movie on the current line and every other movie.
		// So, title is the title on the current line of hte file. curTitle is every other movie title.
		line = bufRead.readLine();
		while (line != null){
		    if ((lineNo < startRange) || (lineNo >= endRange)) {
			lineNo++;
			line = bufRead.readLine();
			continue;
		    }
		    String title = line.substring(0, line.indexOf("@"));
		    line = line.substring(line.indexOf("@")+1, line.length());
		    String[] splitline = line.split(":"); //Splitline length is the number of unique words. Each element has a pair

		    // Reset temp similarity. This array holds the similarity between the active title (from the file) and every other movie title.
		    for (int i = 0; i < 17770; i++) { tempSimilarity[i] = 0.0f; }

		    // For each title, calculate the similarity, store it in tempSimilarity in all relevant places (may be multiple for overlapping titles)
		    for (int titleIndex = 0; titleIndex < movieTitles.size(); titleIndex++) {
			String curTitle = (String)(movieTitles.elementAt(titleIndex));
			double similarityAccum = 0.0;
			
			for (int i = 0; i < splitline.length; i++) {
			    String[] terms = splitline[i].split("@");
			    String index = curTitle+"@"+terms[0]; // hash index is movieTitle + term
			    //			    System.out.println("index: "+index);

			    // Compute cosine distance as A dot B / (length A) * (length B)
			    if (tfidfMap.containsKey(index)) {
				similarityAccum += (Float.parseFloat(terms[1])) * ((Float)(tfidfMap.get(index))).floatValue();	
				//				System.out.println("Matched index: "+index+" "+similarityAccum+" "+((Float)(tfidfMap.get(index))).floatValue()); 
			    }
			}

			// Set the similarity at every matching article (columns)
			float similarityMeasure = (float)similarityAccum / (((Float)(vectorLengths.get(movieTitles.indexOf(title)))).floatValue() * ((Float)(vectorLengths.get(movieTitles.indexOf(curTitle)))).floatValue());

			/*Vector temp = (Vector)(movieTitleMap.get(curTitle));
			if (temp == null) {
			    System.out.println("Can't find "+title+" in the map");
			    continue;
			}
			for (java.util.Iterator it = temp.iterator(); it.hasNext();) {
			    int index = ((Integer)it.next()).intValue();
			    System.out.println("Found movie at index: "+index);
			    tempSimilarity[index] = similarityMeasure;
			    }*/
			for (int i = 0; i < 17770; i++) {
			    if (curTitle.equals(titleArray[i])) {
				tempSimilarity[i] = similarityMeasure;
			    }
			}
		    }
		    String output = ""+title+"$";
		    for (int i = 0; i < 17770; i++) {
			output += " "+tempSimilarity[i];
		    }
		    out.write(output+"\n");
		    out.flush();
		    line = bufRead.readLine();
		    completed++;
		    if (completed % 200 == 0) {
			System.out.println("(start, end) : # completed | ("+startRange+","+endRange+") :"+completed);
		    }
		}
		bufRead.close();
		out.close();
	    
	    } catch (Exception e){
		System.out.println("Error calculating similarities"); e.printStackTrace();
	    }

	}

	public Calculator(int startRange, int endRange) {
	    this.startRange = startRange;
	    this.endRange = endRange;
	    try {
		out = new BufferedWriter(new FileWriter("javasim."+startRange+"."+endRange, false));
	    } catch (IOException e) {
	    }
	}
    }
}
/*
* Copyright (c) 2009 John Lees-Miller, Fraser Anderson
* 
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
* 
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* 
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/

